# encoding: utf-8

import time
import uuid
import datetime
import re
import json
from hashlib import sha1, sha256

import scrapy
from scrapy import Request, FormRequest

try:
    from App_Spd.items import AppSpdItem
    from App_Spd.utils import analysis, get_urlid, get_pubtime, timestamp_now_to
except:
    from app_spider.items import AppSpdItem
    from app_spider.utils import analysis, get_urlid, get_pubtime, timestamp_now_to

headers = {
    'Host': 'proxy.cztvcloud.com',
    'type': 'fusion',
    'user-agent': 'okhttp/4.7.2',
}


class ZhangshangqingyuanAppSpider(scrapy.Spider):
    websiteId = '2141422583'
    pubSource = '掌上庆元'
    name = 'zhangshangqingyuan_app'

    list_url = 'https://proxy.cztvcloud.com/api/fusion/channels/getChannelsInfo?appId=1019&channelId={}&page={}'
    detail_url = 'https://proxy.cztvcloud.com/api/fusion/news/getNewsInfo?appId=1019&newsId={}'
    channles = [
        '749',
        '779','780','2322','2313',
        '1694','778','781','766',
    ]

    def start_requests(self):
        for chl in self.channles:
            for page in range(1, 2):
                yield Request(url=self.list_url.format(chl, page), headers=headers)

    def parse(self, response, **l):
        resp = response.json()
        for ite in resp['data']['block'][0]['items']:
            news_id = ite.get('id')
            if news_id:
                yield Request(url=self.detail_url.format(news_id), callback=self.parse_detail, headers=headers)

    def parse_detail(self, response):
        # print(response.text)
        resp = response.json()
        item = AppSpdItem()
        item['title'] = resp['data'].get('title', '')
        item['author'] = resp['data'].get('authorName', '')
        item['pubtime'] = timestamp_now_to(resp['data'].get('createdAt'))
        item['content'] = analysis(resp['data']['content'])
        item['websiteId'] = 2141422583  # 提供app对应的id
        item['pubSource'] = "掌上庆元"  # 提供app对应名称
        item['url'] = response.url
        item['urlid'] = get_urlid(item['url'])

        for k, v in item.items():
            if not v:
                v = ''
            if isinstance(v, str):
                v = v.strip()
            item[k] = v
        yield item
